{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "#可视化\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "FLAGS = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载。\n",
    "\n",
    "<font color=#ff0000>**这里将data_dir改为适合你的运行环境的目录**</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-1f2e3e29ecda>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting Mnist_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting Mnist_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting Mnist_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting Mnist_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "(55000, 784) (55000, 10)\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = 'Mnist_data/'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "print(mnist.train.images.shape,mnist.train.labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义各层神经元数量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_input_layer = 28 * 28  # 输入层\n",
    "n_output_layer = 10  # 输出层\n",
    "\n",
    "# 层数的选择：线性数据使用1层，非线性数据使用2层, 超级非线性使用3+层。层数、神经元过多会导致过拟合\n",
    "n_layer_1 = 1000  # hide layer\n",
    "n_layer_2 = 1300  # hide layer\n",
    "n_layer_3 = 800  # hide layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义3隐层神经网络\n",
    "def neural_network(x):\n",
    "    # 定义第一层\"神经元\"的权重和biases\n",
    "    layer_1_w_b = {'w_': tf.Variable(tf.truncated_normal([n_input_layer, n_layer_1])),\n",
    "                   'b_': tf.Variable(tf.truncated_normal([n_layer_1]))}\n",
    "    # 定义第二层\"神经元\"的权重和biases\n",
    "    layer_2_w_b = {'w_': tf.Variable(tf.truncated_normal([n_layer_1, n_layer_2])),\n",
    "                   'b_': tf.Variable(tf.truncated_normal([n_layer_2]))}\n",
    "    # 定义第三层\"神经元\"的权重和biases\n",
    "    layer_3_w_b = {'w_': tf.Variable(tf.truncated_normal([n_layer_2, n_layer_3])),\n",
    "                   'b_': tf.Variable(tf.truncated_normal([n_layer_3]))}\n",
    "    # 定义输出层\"神经元\"的权重和biases\n",
    "    layer_output_w_b = {'w_': tf.Variable(tf.truncated_normal([n_layer_3, n_output_layer])),\n",
    "                        'b_': tf.Variable(tf.truncated_normal([n_output_layer]))}\n",
    " \n",
    "    # w·x+b\n",
    "    layer_1 = tf.add(tf.matmul(x, layer_1_w_b['w_']), layer_1_w_b['b_'])\n",
    "    layer_1 = tf.nn.selu(layer_1)  # 激活函数 relu sigmoid  tanh elu  softplus\n",
    "    layer_2 = tf.add(tf.matmul(layer_1, layer_2_w_b['w_']), layer_2_w_b['b_'])\n",
    "    layer_2 = tf.nn.selu(layer_2)  # 激活函数\n",
    "    layer_3 = tf.add(tf.matmul(layer_2, layer_3_w_b['w_']), layer_3_w_b['b_'])\n",
    "    layer_3 = tf.nn.selu(layer_3)  # 激活函数\n",
    "    layer_output = tf.add(tf.matmul(layer_3, layer_output_w_b['w_']), layer_output_w_b['b_'])\n",
    " \n",
    "    return layer_output, layer_1_w_b, layer_2_w_b, layer_3_w_b, layer_output_w_b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 每次使用100条数据进行训练\n",
    "batch_size = 100\n",
    "\n",
    "x = tf.placeholder(dtype=tf.float32, shape=[None, n_input_layer])   # 输入数据占位符\n",
    "y = tf.placeholder(dtype=tf.float32, shape=[None, n_output_layer])   # 输出数据占位符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用数据训练神经网络\n",
    "def train_neural_network(X, Y, epochs=10):\n",
    "    #创建神经网络\n",
    "    y, layer_1_w_b, layer_2_w_b, layer_3_w_b, layer_output_w_b = neural_network(X)\n",
    "    #代价函数\n",
    "#     cost_func = tf.reduce_sum(tf.nn.softmax_cross_entropy_with_logits(logits= y, labels=Y))\n",
    "    cost_func = tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=tf.argmax(Y,1), logits= y))\n",
    "    #惩罚因子\n",
    "    regularization_rate = 0.1\n",
    "    #计算L2正则化损失函数\n",
    "    regularizer = tf.contrib.layers.l2_regularizer(regularization_rate)\n",
    "    #将变量的L2正则化损失添加到集合中\n",
    "    tf.add_to_collection(\"losses\",regularizer(layer_1_w_b['w_']))\n",
    "    tf.add_to_collection(\"losses\",regularizer(layer_2_w_b['w_']))\n",
    "    tf.add_to_collection(\"losses\",regularizer(layer_3_w_b['w_']))\n",
    "    tf.add_to_collection(\"losses\",regularizer(layer_output_w_b['w_']))\n",
    "    #将代价函数与正则项合并\n",
    "    tf.add_to_collection(\"losses\",cost_func)\n",
    "    #获取整个模型的损失函数,tf.get_collection(\"losses\")返回集合中定义的损失\n",
    "    #将整个集合中的损失相加得到整个模型的损失函数\n",
    "    loss = tf.add_n(tf.get_collection(\"losses\"))\n",
    "\n",
    "    #学习速率，随迭代次数进行递减\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "    #设置基础学习率\n",
    "    starter_learning_rate = 0.0005\n",
    "    #设置学习率的衰减率\n",
    "    learning_rate_decay = 0.95\n",
    "    #设置指数衰减学习率\n",
    "    learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, mnist.train.num_examples/batch_size/2, learning_rate_decay)\n",
    "    \n",
    "    train_step = tf.train.RMSPropOptimizer(learning_rate).minimize(loss)  \n",
    "    # AdamOptimizer RMSPropOptimizer 表现最好\n",
    "    # AdamOptimizer GradientDescentOptimizer AdadeltaOptimizer FtrlOptimizer ProximalGradientDescentOptimizer \n",
    "    # ProximalAdagradOptimizer RMSPropOptimizer\n",
    "    \n",
    "    #每迭代一次需要更新神经网络中的参数\n",
    "    train_op = tf.group(train_step)\n",
    " \n",
    "    with tf.Session() as session:\n",
    "        session.run(tf.initialize_all_variables())\n",
    "        \n",
    "        correct = tf.equal(tf.argmax(y, 1), tf.argmax(Y, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct, 'float'))\n",
    "        \n",
    "        epoch_loss = 0\n",
    "        epoch_loss_mat = []\n",
    "        epoch_accuracy_mat = []\n",
    "        epoch_accuracy_t_mat = []\n",
    "        for epoch in range(epochs):\n",
    "            for i in range(int(mnist.train.num_examples / batch_size)):\n",
    "                batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "                _, cost_vel = session.run([train_op, loss], feed_dict={X: batch_xs, Y: batch_ys, global_step:i*(epoch+1)})\n",
    "                epoch_loss += cost_vel\n",
    "            epoch_accuracy = accuracy.eval({X: mnist.train.images, Y: mnist.train.labels})\n",
    "            epoch_accuracy_t = accuracy.eval({X: mnist.test.images, Y: mnist.test.labels})\n",
    "            print('epoch ', epoch, ' loss      : ', epoch_loss)\n",
    "            print('epoch ', epoch, ' accuracy  : ', epoch_accuracy)\n",
    "            print('epoch ', epoch, ' accuracy_t: ', epoch_accuracy_t)\n",
    "            epoch_loss_mat.append(epoch_loss)\n",
    "            epoch_accuracy_mat.append(epoch_accuracy)\n",
    "            epoch_accuracy_t_mat.append(epoch_accuracy_t)\n",
    "            epoch_loss = 0\n",
    " \n",
    "        \n",
    "        print('训练准确率: ', accuracy.eval({X: mnist.train.images, Y: mnist.train.labels}))\n",
    "        print('测试准确率: ', accuracy.eval({X: mnist.test.images, Y: mnist.test.labels}))\n",
    "        \n",
    "        plt.plot(range(epochs), epoch_loss_mat) \n",
    "        plt.xlabel('epochs')\n",
    "        plt.ylabel('epoch_loss')\n",
    "        plt.show() \n",
    "        \n",
    "        plt.plot(range(epochs), epoch_accuracy_mat, label='Train Accuracy') \n",
    "        plt.plot(range(epochs), epoch_accuracy_t_mat, label='Test Accuracy')\n",
    "        plt.legend()\n",
    "        plt.xlabel('epochs')\n",
    "        plt.ylabel('accuracy')\n",
    "        plt.show() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Only call `sparse_softmax_cross_entropy_with_logits` with named arguments (labels=..., logits=..., ...)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-89e4ed05009e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtrain_neural_network\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m20\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-6-d4fed87c6fcf>\u001b[0m in \u001b[0;36mtrain_neural_network\u001b[1;34m(X, Y, epochs)\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[1;31m#代价函数\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;31m#     cost_func = tf.reduce_sum(tf.nn.softmax_cross_entropy_with_logits(logits= y, labels=Y))\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m     \u001b[0mcost_func\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreduce_sum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msparse_softmax_cross_entropy_with_logits\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mY\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogits\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      8\u001b[0m     \u001b[1;31m#惩罚因子\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[0mregularization_rate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0.1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\nn_ops.py\u001b[0m in \u001b[0;36msparse_softmax_cross_entropy_with_logits\u001b[1;34m(_sentinel, labels, logits, name)\u001b[0m\n\u001b[0;32m   2022\u001b[0m   \"\"\"\n\u001b[0;32m   2023\u001b[0m   _ensure_xent_args(\"sparse_softmax_cross_entropy_with_logits\", _sentinel,\n\u001b[1;32m-> 2024\u001b[1;33m                     labels, logits)\n\u001b[0m\u001b[0;32m   2025\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2026\u001b[0m   \u001b[1;31m# TODO(pcmurray) Raise an error when the label is not an index in\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\nn_ops.py\u001b[0m in \u001b[0;36m_ensure_xent_args\u001b[1;34m(name, sentinel, labels, logits)\u001b[0m\n\u001b[0;32m   1773\u001b[0m   \u001b[1;32mif\u001b[0m \u001b[0msentinel\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1774\u001b[0m     raise ValueError(\"Only call `%s` with \"\n\u001b[1;32m-> 1775\u001b[1;33m                      \"named arguments (labels=..., logits=..., ...)\" % name)\n\u001b[0m\u001b[0;32m   1776\u001b[0m   \u001b[1;32mif\u001b[0m \u001b[0mlabels\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mlogits\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1777\u001b[0m     \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Both labels and logits must be provided.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Only call `sparse_softmax_cross_entropy_with_logits` with named arguments (labels=..., logits=..., ...)"
     ]
    }
   ],
   "source": [
    "train_neural_network(X=x, Y=y, epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到92%左右的准确率。\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至98%以上的准确率。\n",
    "Hint：\n",
    "- 多隐层\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 隐层神经元数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
